import scrapy

from lj.items import LjItem


class LianjiaSpider(scrapy.Spider):
    name = 'lianjia'
    allowed_domains = ['lianjia.com']
    start_urls = ['https://nj.fang.lianjia.com/loupan/pg1/?_t=1']
    # start_urls = ['http://lianjia.com/']

    def parse(self, response):
        data = response.json().get("data")
        total = data.get("total")
        page_num = (int(total) // 10) + 1
        page = data.get("selected").get("pager").get("page")
        yield from self.parse_items(data.get("list"))
        print(f"获取第{page}数据完成 {response.url}")
        if page <= page_num:
            num = page + 1
            _url = f"https://nj.fang.lianjia.com/loupan/pg{num}/?_t=1"
            yield scrapy.Request(url=_url)

    @staticmethod
    def parse_items(data: list):
        for i in data:
            item = LjItem()
            item["title"] = i.get("title")
            item["cover_pic"] = i.get("cover_pic")
            item["city_name"] = i.get("city_name")
            item["district"] = i.get("district")
            item["address_remark"] = i.get("address_remark")
            item["house_type"] = i.get("house_type")
            item["average_price"] = i.get("average_price")
            item["show_price_info"] = i.get("show_price_info")

            yield item


